{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型结构\n",
    "\n",
    "- 两层单向LSTM, 输出序列结果, 即(batch_size, step_size, feature_size)\n",
    "- 分别输入到1, 2, 3, 4, 5, 6共6个不同长度的卷积层中\n",
    "- 卷积层为单层\n",
    "- 对于每个问题, 将所有卷积核结果并起来\n",
    "- 将两个问题并起来的结果, 分别[相减并取绝对值], [相乘], 再将所有的结果合并\n",
    "- 输出到两层Dense层中, 最后一层为sigmoid激活函数, 输出结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((10001, 300), (3049, 300))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATA_PATH = \"./data/\"\n",
    "TRAIN_PATH = DATA_PATH + \"train.csv\"\n",
    "TEST_PATH = DATA_PATH + \"test.csv\"\n",
    "WORD_EMBED_PATH = DATA_PATH + \"word_embed.txt\"\n",
    "CHAR_EMBED_PATH = DATA_PATH + \"char_embed.txt\"\n",
    "QUEST_PATH = DATA_PATH + \"question.csv\"\n",
    "\n",
    "train_data = pd.read_csv(TRAIN_PATH)\n",
    "test_data = pd.read_csv(TEST_PATH)\n",
    "question_data = pd.read_csv(QUEST_PATH)\n",
    "word_embedding_data = pd.read_csv(WORD_EMBED_PATH, delimiter=\" \", header=None, index_col=0)\n",
    "char_embedding_data = pd.read_csv(CHAR_EMBED_PATH, delimiter=\" \", header=None, index_col=0)\n",
    "\n",
    "question_data[\"words\"] = question_data[\"words\"].str.split(\" \")\n",
    "question_data[\"chars\"] = question_data[\"chars\"].str.split(\" \")\n",
    "\n",
    "label = train_data[\"label\"].values\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "MAX_COUNT = 10000\n",
    "\n",
    "word_tokenizer = Tokenizer(MAX_COUNT)\n",
    "word_tokenizer.fit_on_texts(question_data[\"words\"])\n",
    "\n",
    "word_embedding_data = np.concatenate(\n",
    "    (\n",
    "        np.zeros(shape=(1, word_embedding_data.shape[1]), dtype=np.float64),\n",
    "        word_embedding_data.loc[list(word_tokenizer.word_index.keys())[:MAX_COUNT]].values\n",
    "    ),\n",
    "    axis=0\n",
    ")\n",
    "\n",
    "char_tokenizer = Tokenizer(MAX_COUNT)\n",
    "char_tokenizer.fit_on_texts(question_data[\"chars\"])\n",
    "\n",
    "char_embedding_data = np.concatenate(\n",
    "    (\n",
    "        np.zeros(shape=(1, char_embedding_data.shape[1]), dtype=np.float64),\n",
    "        char_embedding_data.loc[list(char_tokenizer.word_index.keys())[:MAX_COUNT]].values\n",
    "    ),\n",
    "    axis=0\n",
    ")\n",
    "\n",
    "word_embedding_data.shape, char_embedding_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((254386, 30),\n",
       " (254386, 30),\n",
       " (172956, 30),\n",
       " (172956, 30),\n",
       " (254386, 30),\n",
       " (254386, 30),\n",
       " (172956, 30),\n",
       " (172956, 30))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "SEQ_LEN = 25\n",
    "\n",
    "def gen_word_data(data):\n",
    "    seq_word1 = word_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q1\", right_on=\"qid\")[\"words\"])\n",
    "    seq_word2 = word_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q2\", right_on=\"qid\")[\"words\"])\n",
    "    return pad_sequences(seq_word1, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\"), \\\n",
    "        pad_sequences(seq_word2, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\")\n",
    "    \n",
    "def gen_char_data(data):\n",
    "    seq_char1 = char_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q1\", right_on=\"qid\")[\"chars\"])\n",
    "    seq_char2 = char_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q2\", right_on=\"qid\")[\"chars\"])\n",
    "    return pad_sequences(seq_char1, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\"), \\\n",
    "        pad_sequences(seq_char2, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\")\n",
    "\n",
    "word1, word2 = gen_word_data(train_data)\n",
    "char1, char2 = gen_char_data(train_data)\n",
    "test_word1, test_word2 = gen_word_data(test_data)\n",
    "test_char1, test_char2 = gen_char_data(test_data)\n",
    "\n",
    "word1.shape, word2.shape, test_word1.shape, test_word2.shape, char1.shape, char2.shape, test_char1.shape, test_char2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.layers import LSTM, Bidirectional, TimeDistributed\n",
    "from keras.layers import Conv1D, MaxPool1D, GlobalAveragePooling1D\n",
    "from keras.layers import Input, Embedding, Dropout, BatchNormalization, Dense, Flatten, Lambda, K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# general\n",
    "NUM_EPOCHES = 30\n",
    "EPOCHES1 = 3\n",
    "EPOCHES2 = 5\n",
    "EPOCHES3 = 22\n",
    "BATCH_SIZE = 1024\n",
    "DROP_RATE = 0.3\n",
    "PATIENCE = 6\n",
    "\n",
    "# cnn\n",
    "CONV_LEN1 = 128\n",
    "CONV_LEN2 = 128\n",
    "CONV_LEN3 = 128\n",
    "CONV_LEN4 = 128\n",
    "CONV_LEN5 = 128\n",
    "CONV_LEN6 = 128\n",
    "CONV_LEN = CONV_LEN1 + CONV_LEN2 + CONV_LEN3 + CONV_LEN4 + CONV_LEN5 + CONV_LEN6\n",
    "\n",
    "# lstm\n",
    "LSTM_SIZE1 = 256\n",
    "LSTM_SIZE2 = 256\n",
    "LSTM_DROP_RATE = 0.3\n",
    "\n",
    "# dense\n",
    "DENSE_SIZE1 = 512\n",
    "DENSE_SIZE2 = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cnn_layer1(inputa, inputb, filters, kernel_size):\n",
    "    conv = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\", activation=\"relu\")\n",
    "    conv_outputa = conv(inputa)\n",
    "    conv_outputa = GlobalAveragePooling1D()(conv_outputa)\n",
    "    conv_outputb = conv(inputb)\n",
    "    conv_outputb = GlobalAveragePooling1D()(conv_outputb)\n",
    "    return conv_outputa, conv_outputb\n",
    "    \n",
    "def cnn_layer2(inputa, inputb, filters, kernel_size):\n",
    "    conv = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\", activation=\"relu\")\n",
    "    conv_outputa = conv(inputa)\n",
    "    conv_outputa = MaxPool1D(pool_size=SEQ_LEN)(conv_outputa)\n",
    "    conv_outputa = Flatten()(conv_outputa)\n",
    "    conv_outputb = conv(inputb)\n",
    "    conv_outputb = MaxPool1D(pool_size=SEQ_LEN)(conv_outputb)\n",
    "    conv_outputb = Flatten()(conv_outputb)\n",
    "    return conv_outputa, conv_outputb\n",
    "\n",
    "def cnn_layer3(inputa, inputb, filters, kernel_size):\n",
    "    conv = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\", activation=\"relu\")\n",
    "    \n",
    "    conv_outputa = conv(inputa)\n",
    "    conv_outputa1 = Flatten()(MaxPool1D(pool_size=SEQ_LEN)(conv_outputa))\n",
    "    conv_outputa2 = GlobalAveragePooling1D()(conv_outputa)\n",
    "    conv_outputa = concatenate([conv_outputa1, conv_outputa2])\n",
    "    \n",
    "    conv_outputb = conv(inputb)\n",
    "    conv_outputb1 = Flatten()(MaxPool1D(pool_size=SEQ_LEN)(conv_outputb))\n",
    "    conv_outputb2 = GlobalAveragePooling1D()(conv_outputb)\n",
    "    conv_outputb = concatenate([conv_outputb1, conv_outputb2])\n",
    "    \n",
    "    return conv_outputa, conv_outputb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# WORDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py:497: calling conv1d (from tensorflow.python.ops.nn_ops) with data_format=NHWC is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "`NHWC` for data_format is deprecated, use `NWC` instead\n",
      "Train on 228946 samples, validate on 25440 samples\n",
      "Epoch 1/30\n",
      "228946/228946 [==============================] - 167s 729us/step - loss: 0.4562 - acc: 0.7857 - val_loss: 0.3437 - val_acc: 0.8555\n",
      "Epoch 2/30\n",
      "228946/228946 [==============================] - 162s 707us/step - loss: 0.3170 - acc: 0.8602 - val_loss: 0.2723 - val_acc: 0.8814\n",
      "Epoch 3/30\n",
      "228946/228946 [==============================] - 159s 694us/step - loss: 0.2703 - acc: 0.8825 - val_loss: 0.2346 - val_acc: 0.9001\n",
      "Epoch 4/30\n",
      "228946/228946 [==============================] - 164s 714us/step - loss: 0.2431 - acc: 0.8959 - val_loss: 0.2552 - val_acc: 0.8983\n",
      "Epoch 5/30\n",
      "228946/228946 [==============================] - 165s 722us/step - loss: 0.2257 - acc: 0.9041 - val_loss: 0.2271 - val_acc: 0.9105\n",
      "Epoch 6/30\n",
      "228946/228946 [==============================] - 170s 741us/step - loss: 0.2109 - acc: 0.9109 - val_loss: 0.2069 - val_acc: 0.9182\n",
      "Epoch 7/30\n",
      "228946/228946 [==============================] - 158s 692us/step - loss: 0.2017 - acc: 0.9153 - val_loss: 0.1986 - val_acc: 0.9187\n",
      "Epoch 8/30\n",
      "228946/228946 [==============================] - 168s 732us/step - loss: 0.1918 - acc: 0.9204 - val_loss: 0.1966 - val_acc: 0.9209\n",
      "Epoch 9/30\n",
      "228946/228946 [==============================] - 167s 730us/step - loss: 0.1859 - acc: 0.9226 - val_loss: 0.2010 - val_acc: 0.9215\n",
      "Epoch 10/30\n",
      "228946/228946 [==============================] - 166s 727us/step - loss: 0.1803 - acc: 0.9253 - val_loss: 0.2055 - val_acc: 0.9198\n",
      "Epoch 11/30\n",
      "228946/228946 [==============================] - 168s 734us/step - loss: 0.1745 - acc: 0.9274 - val_loss: 0.1898 - val_acc: 0.9242\n",
      "Epoch 12/30\n",
      "228946/228946 [==============================] - 167s 731us/step - loss: 0.1699 - acc: 0.9291 - val_loss: 0.1986 - val_acc: 0.9233\n",
      "Epoch 13/30\n",
      "228946/228946 [==============================] - 164s 718us/step - loss: 0.1653 - acc: 0.9318 - val_loss: 0.1902 - val_acc: 0.9258\n",
      "Epoch 14/30\n",
      "228946/228946 [==============================] - 164s 714us/step - loss: 0.1620 - acc: 0.9328 - val_loss: 0.1875 - val_acc: 0.9282\n",
      "Epoch 15/30\n",
      "228946/228946 [==============================] - 163s 712us/step - loss: 0.1584 - acc: 0.9344 - val_loss: 0.1840 - val_acc: 0.9292\n",
      "Epoch 16/30\n",
      "228946/228946 [==============================] - 163s 714us/step - loss: 0.1556 - acc: 0.9360 - val_loss: 0.1954 - val_acc: 0.9265\n",
      "Epoch 17/30\n",
      "228946/228946 [==============================] - 163s 713us/step - loss: 0.1534 - acc: 0.9365 - val_loss: 0.1897 - val_acc: 0.9263\n",
      "Epoch 18/30\n",
      "228946/228946 [==============================] - 164s 717us/step - loss: 0.1511 - acc: 0.9375 - val_loss: 0.1852 - val_acc: 0.9285\n",
      "Epoch 19/30\n",
      "228946/228946 [==============================] - 165s 720us/step - loss: 0.1485 - acc: 0.9389 - val_loss: 0.1916 - val_acc: 0.9278\n",
      "Epoch 20/30\n",
      "228946/228946 [==============================] - 164s 716us/step - loss: 0.1449 - acc: 0.9403 - val_loss: 0.1955 - val_acc: 0.9269\n",
      "Epoch 21/30\n",
      "228946/228946 [==============================] - 164s 714us/step - loss: 0.1441 - acc: 0.9409 - val_loss: 0.1908 - val_acc: 0.9294\n",
      "Epoch 22/30\n",
      "228946/228946 [==============================] - 163s 712us/step - loss: 0.1409 - acc: 0.9420 - val_loss: 0.1934 - val_acc: 0.9279\n",
      "Epoch 23/30\n",
      "228946/228946 [==============================] - 167s 727us/step - loss: 0.1397 - acc: 0.9418 - val_loss: 0.1935 - val_acc: 0.9274\n",
      "load model ./log/20180707-200651.Multi_LSTM_CNN_v1.word.015.hdf5\n",
      "Train on 228946 samples, validate on 25440 samples\n",
      "Epoch 1/30\n",
      "228946/228946 [==============================] - 167s 727us/step - loss: 0.4526 - acc: 0.7859 - val_loss: 0.3311 - val_acc: 0.8631\n",
      "Epoch 2/30\n",
      "228946/228946 [==============================] - 164s 717us/step - loss: 0.3147 - acc: 0.8605 - val_loss: 0.2657 - val_acc: 0.8897\n",
      "Epoch 3/30\n",
      "228946/228946 [==============================] - 161s 705us/step - loss: 0.2696 - acc: 0.8830 - val_loss: 0.2417 - val_acc: 0.8988\n",
      "Epoch 4/30\n",
      "228946/228946 [==============================] - 167s 729us/step - loss: 0.2421 - acc: 0.8960 - val_loss: 0.2226 - val_acc: 0.9085\n",
      "Epoch 5/30\n",
      "228946/228946 [==============================] - 168s 733us/step - loss: 0.2253 - acc: 0.9042 - val_loss: 0.2174 - val_acc: 0.9120\n",
      "Epoch 6/30\n",
      "228946/228946 [==============================] - 167s 729us/step - loss: 0.2107 - acc: 0.9111 - val_loss: 0.2059 - val_acc: 0.9172\n",
      "Epoch 7/30\n",
      "228946/228946 [==============================] - 166s 726us/step - loss: 0.2020 - acc: 0.9150 - val_loss: 0.2172 - val_acc: 0.9129\n",
      "Epoch 8/30\n",
      "228946/228946 [==============================] - 167s 727us/step - loss: 0.1931 - acc: 0.9194 - val_loss: 0.1902 - val_acc: 0.9230\n",
      "Epoch 9/30\n",
      "228946/228946 [==============================] - 166s 727us/step - loss: 0.1857 - acc: 0.9224 - val_loss: 0.1871 - val_acc: 0.9247\n",
      "Epoch 10/30\n",
      "228946/228946 [==============================] - 166s 724us/step - loss: 0.1806 - acc: 0.9248 - val_loss: 0.1903 - val_acc: 0.9250\n",
      "Epoch 11/30\n",
      "228946/228946 [==============================] - 166s 726us/step - loss: 0.1746 - acc: 0.9278 - val_loss: 0.1797 - val_acc: 0.9271\n",
      "Epoch 12/30\n",
      "228946/228946 [==============================] - 168s 732us/step - loss: 0.1692 - acc: 0.9295 - val_loss: 0.1954 - val_acc: 0.9247\n",
      "Epoch 13/30\n",
      "228946/228946 [==============================] - 167s 729us/step - loss: 0.1664 - acc: 0.9311 - val_loss: 0.1848 - val_acc: 0.9259\n",
      "Epoch 14/30\n",
      "228946/228946 [==============================] - 167s 729us/step - loss: 0.1624 - acc: 0.9324 - val_loss: 0.1836 - val_acc: 0.9261\n",
      "Epoch 15/30\n",
      "228946/228946 [==============================] - 166s 724us/step - loss: 0.1581 - acc: 0.9349 - val_loss: 0.1804 - val_acc: 0.9292\n",
      "Epoch 16/30\n",
      "228946/228946 [==============================] - 167s 728us/step - loss: 0.1564 - acc: 0.9353 - val_loss: 0.1794 - val_acc: 0.9297\n",
      "Epoch 17/30\n",
      "228946/228946 [==============================] - 163s 711us/step - loss: 0.1546 - acc: 0.9360 - val_loss: 0.1827 - val_acc: 0.9294\n",
      "Epoch 18/30\n",
      "228946/228946 [==============================] - 167s 731us/step - loss: 0.1500 - acc: 0.9374 - val_loss: 0.1736 - val_acc: 0.9329\n",
      "Epoch 19/30\n",
      "228946/228946 [==============================] - 166s 727us/step - loss: 0.1490 - acc: 0.9390 - val_loss: 0.1842 - val_acc: 0.9281\n",
      "Epoch 20/30\n",
      "228946/228946 [==============================] - 166s 727us/step - loss: 0.1467 - acc: 0.9396 - val_loss: 0.1924 - val_acc: 0.9309\n",
      "Epoch 21/30\n",
      "228946/228946 [==============================] - 168s 733us/step - loss: 0.1440 - acc: 0.9404 - val_loss: 0.1851 - val_acc: 0.9303\n",
      "Epoch 22/30\n",
      "228946/228946 [==============================] - 166s 727us/step - loss: 0.1430 - acc: 0.9409 - val_loss: 0.1836 - val_acc: 0.9313\n",
      "Epoch 23/30\n",
      "228946/228946 [==============================] - 166s 727us/step - loss: 0.1401 - acc: 0.9424 - val_loss: 0.1820 - val_acc: 0.9322\n",
      "Epoch 24/30\n",
      "228946/228946 [==============================] - 166s 726us/step - loss: 0.1382 - acc: 0.9425 - val_loss: 0.1819 - val_acc: 0.9312\n",
      "Epoch 25/30\n",
      "228946/228946 [==============================] - 166s 723us/step - loss: 0.1354 - acc: 0.9437 - val_loss: 0.1914 - val_acc: 0.9270\n",
      "Epoch 26/30\n",
      "228946/228946 [==============================] - 166s 727us/step - loss: 0.1343 - acc: 0.9444 - val_loss: 0.1749 - val_acc: 0.9341\n",
      "load model ./log/20180707-211124.Multi_LSTM_CNN_v1.word.018.hdf5\n",
      "Train on 228947 samples, validate on 25439 samples\n",
      "Epoch 1/30\n",
      "228947/228947 [==============================] - 167s 729us/step - loss: 0.4492 - acc: 0.7890 - val_loss: 0.3311 - val_acc: 0.8628\n",
      "Epoch 2/30\n",
      "228947/228947 [==============================] - 160s 698us/step - loss: 0.3123 - acc: 0.8627 - val_loss: 0.2702 - val_acc: 0.8816\n",
      "Epoch 3/30\n",
      "228947/228947 [==============================] - 166s 727us/step - loss: 0.2684 - acc: 0.8843 - val_loss: 0.2380 - val_acc: 0.8994\n",
      "Epoch 4/30\n",
      "228947/228947 [==============================] - 166s 727us/step - loss: 0.2418 - acc: 0.8970 - val_loss: 0.2237 - val_acc: 0.9075\n",
      "Epoch 5/30\n",
      "228947/228947 [==============================] - 167s 731us/step - loss: 0.2252 - acc: 0.9045 - val_loss: 0.2105 - val_acc: 0.9139\n",
      "Epoch 6/30\n",
      "228947/228947 [==============================] - 166s 724us/step - loss: 0.2116 - acc: 0.9103 - val_loss: 0.2093 - val_acc: 0.9155\n",
      "Epoch 7/30\n",
      "228947/228947 [==============================] - 168s 733us/step - loss: 0.2018 - acc: 0.9149 - val_loss: 0.2067 - val_acc: 0.9156\n",
      "Epoch 8/30\n",
      "228947/228947 [==============================] - 166s 725us/step - loss: 0.1937 - acc: 0.9193 - val_loss: 0.1913 - val_acc: 0.9222\n",
      "Epoch 9/30\n",
      "228947/228947 [==============================] - 167s 729us/step - loss: 0.1863 - acc: 0.9223 - val_loss: 0.1992 - val_acc: 0.9225\n",
      "Epoch 10/30\n",
      "228947/228947 [==============================] - 166s 726us/step - loss: 0.1804 - acc: 0.9254 - val_loss: 0.1962 - val_acc: 0.9237\n",
      "Epoch 11/30\n",
      "228947/228947 [==============================] - 166s 726us/step - loss: 0.1755 - acc: 0.9274 - val_loss: 0.1998 - val_acc: 0.9227\n",
      "Epoch 12/30\n",
      "228947/228947 [==============================] - 166s 727us/step - loss: 0.1714 - acc: 0.9289 - val_loss: 0.1910 - val_acc: 0.9261\n",
      "Epoch 13/30\n",
      "228947/228947 [==============================] - 164s 716us/step - loss: 0.1665 - acc: 0.9312 - val_loss: 0.2045 - val_acc: 0.9208\n",
      "Epoch 14/30\n",
      "228947/228947 [==============================] - 167s 731us/step - loss: 0.1638 - acc: 0.9325 - val_loss: 0.1905 - val_acc: 0.9265\n",
      "Epoch 15/30\n",
      "228947/228947 [==============================] - 165s 721us/step - loss: 0.1600 - acc: 0.9335 - val_loss: 0.1881 - val_acc: 0.9268\n",
      "Epoch 16/30\n",
      "228947/228947 [==============================] - 166s 726us/step - loss: 0.1567 - acc: 0.9350 - val_loss: 0.1911 - val_acc: 0.9267\n",
      "Epoch 17/30\n",
      "228947/228947 [==============================] - 167s 731us/step - loss: 0.1530 - acc: 0.9367 - val_loss: 0.1878 - val_acc: 0.9287\n",
      "Epoch 18/30\n",
      "228947/228947 [==============================] - 163s 714us/step - loss: 0.1507 - acc: 0.9377 - val_loss: 0.1913 - val_acc: 0.9273\n",
      "Epoch 19/30\n",
      "228947/228947 [==============================] - 166s 724us/step - loss: 0.1497 - acc: 0.9384 - val_loss: 0.1855 - val_acc: 0.9287\n",
      "Epoch 20/30\n",
      "228947/228947 [==============================] - 166s 726us/step - loss: 0.1476 - acc: 0.9387 - val_loss: 0.1972 - val_acc: 0.9263\n",
      "Epoch 21/30\n",
      "228947/228947 [==============================] - 167s 728us/step - loss: 0.1440 - acc: 0.9408 - val_loss: 0.1842 - val_acc: 0.9300\n",
      "Epoch 22/30\n",
      "228947/228947 [==============================] - 162s 708us/step - loss: 0.1429 - acc: 0.9415 - val_loss: 0.1845 - val_acc: 0.9290\n",
      "Epoch 23/30\n",
      "228947/228947 [==============================] - 168s 734us/step - loss: 0.1407 - acc: 0.9421 - val_loss: 0.1874 - val_acc: 0.9306\n",
      "Epoch 24/30\n",
      "228947/228947 [==============================] - 167s 728us/step - loss: 0.1392 - acc: 0.9429 - val_loss: 0.1860 - val_acc: 0.9267\n",
      "Epoch 25/30\n",
      "228947/228947 [==============================] - 168s 732us/step - loss: 0.1356 - acc: 0.9443 - val_loss: 0.1968 - val_acc: 0.9268\n",
      "Epoch 26/30\n",
      "228947/228947 [==============================] - 166s 727us/step - loss: 0.1348 - acc: 0.9451 - val_loss: 0.1896 - val_acc: 0.9293\n",
      "Epoch 27/30\n",
      "228947/228947 [==============================] - 168s 732us/step - loss: 0.1341 - acc: 0.9451 - val_loss: 0.1903 - val_acc: 0.9286\n",
      "Epoch 28/30\n",
      "228947/228947 [==============================] - 167s 729us/step - loss: 0.1309 - acc: 0.9461 - val_loss: 0.2040 - val_acc: 0.9263\n",
      "Epoch 29/30\n",
      "228947/228947 [==============================] - 168s 732us/step - loss: 0.1298 - acc: 0.9465 - val_loss: 0.1959 - val_acc: 0.9277\n",
      "load model ./log/20180707-222455.Multi_LSTM_CNN_v1.word.021.hdf5\n",
      "Train on 228947 samples, validate on 25439 samples\n",
      "Epoch 1/30\n",
      "228947/228947 [==============================] - 163s 712us/step - loss: 0.4576 - acc: 0.7845 - val_loss: 0.3456 - val_acc: 0.8527\n",
      "Epoch 2/30\n",
      "228947/228947 [==============================] - 165s 722us/step - loss: 0.3178 - acc: 0.8588 - val_loss: 0.2798 - val_acc: 0.8817\n",
      "Epoch 3/30\n",
      "228947/228947 [==============================] - 166s 724us/step - loss: 0.2707 - acc: 0.8823 - val_loss: 0.2356 - val_acc: 0.8994\n",
      "Epoch 4/30\n",
      "228947/228947 [==============================] - 167s 731us/step - loss: 0.2442 - acc: 0.8956 - val_loss: 0.2204 - val_acc: 0.9093\n",
      "Epoch 5/30\n",
      "228947/228947 [==============================] - 166s 727us/step - loss: 0.2268 - acc: 0.9045 - val_loss: 0.2211 - val_acc: 0.9097\n",
      "Epoch 6/30\n",
      "228947/228947 [==============================] - 166s 727us/step - loss: 0.2137 - acc: 0.9095 - val_loss: 0.2102 - val_acc: 0.9155\n",
      "Epoch 7/30\n",
      "228947/228947 [==============================] - 164s 718us/step - loss: 0.2034 - acc: 0.9146 - val_loss: 0.1934 - val_acc: 0.9226\n",
      "Epoch 8/30\n",
      "228947/228947 [==============================] - 163s 713us/step - loss: 0.1947 - acc: 0.9184 - val_loss: 0.1928 - val_acc: 0.9229\n",
      "Epoch 9/30\n",
      "228947/228947 [==============================] - 165s 721us/step - loss: 0.1879 - acc: 0.9223 - val_loss: 0.1957 - val_acc: 0.9217\n",
      "Epoch 10/30\n",
      "228947/228947 [==============================] - 165s 722us/step - loss: 0.1813 - acc: 0.9247 - val_loss: 0.1898 - val_acc: 0.9269\n",
      "Epoch 11/30\n",
      "228947/228947 [==============================] - 163s 711us/step - loss: 0.1768 - acc: 0.9265 - val_loss: 0.1940 - val_acc: 0.9246\n",
      "Epoch 12/30\n",
      "228947/228947 [==============================] - 166s 723us/step - loss: 0.1727 - acc: 0.9286 - val_loss: 0.2035 - val_acc: 0.9239\n",
      "Epoch 13/30\n",
      "228947/228947 [==============================] - 166s 726us/step - loss: 0.1675 - acc: 0.9305 - val_loss: 0.1801 - val_acc: 0.9303\n",
      "Epoch 14/30\n",
      "228947/228947 [==============================] - 164s 718us/step - loss: 0.1645 - acc: 0.9322 - val_loss: 0.1804 - val_acc: 0.9294\n",
      "Epoch 15/30\n",
      "228947/228947 [==============================] - 165s 723us/step - loss: 0.1613 - acc: 0.9335 - val_loss: 0.1794 - val_acc: 0.9306\n",
      "Epoch 16/30\n",
      "228947/228947 [==============================] - 168s 733us/step - loss: 0.1580 - acc: 0.9348 - val_loss: 0.1875 - val_acc: 0.9297\n",
      "Epoch 17/30\n",
      "228947/228947 [==============================] - 167s 729us/step - loss: 0.1542 - acc: 0.9362 - val_loss: 0.1854 - val_acc: 0.9317\n",
      "Epoch 18/30\n",
      "228947/228947 [==============================] - 166s 727us/step - loss: 0.1530 - acc: 0.9365 - val_loss: 0.1716 - val_acc: 0.9332\n",
      "Epoch 19/30\n",
      "228947/228947 [==============================] - 167s 729us/step - loss: 0.1499 - acc: 0.9382 - val_loss: 0.1800 - val_acc: 0.9313\n",
      "Epoch 20/30\n",
      "228947/228947 [==============================] - 167s 729us/step - loss: 0.1480 - acc: 0.9387 - val_loss: 0.1811 - val_acc: 0.9286\n",
      "Epoch 21/30\n",
      "228947/228947 [==============================] - 166s 724us/step - loss: 0.1461 - acc: 0.9396 - val_loss: 0.1816 - val_acc: 0.9314\n",
      "Epoch 22/30\n",
      "228947/228947 [==============================] - 166s 725us/step - loss: 0.1436 - acc: 0.9410 - val_loss: 0.1824 - val_acc: 0.9319\n",
      "Epoch 23/30\n",
      "228947/228947 [==============================] - 167s 729us/step - loss: 0.1395 - acc: 0.9426 - val_loss: 0.1802 - val_acc: 0.9324\n",
      "Epoch 24/30\n",
      "228947/228947 [==============================] - 165s 720us/step - loss: 0.1392 - acc: 0.9427 - val_loss: 0.1816 - val_acc: 0.9329\n",
      "Epoch 25/30\n",
      "228947/228947 [==============================] - 166s 726us/step - loss: 0.1371 - acc: 0.9434 - val_loss: 0.1846 - val_acc: 0.9333\n",
      "Epoch 26/30\n",
      "228947/228947 [==============================] - 167s 729us/step - loss: 0.1355 - acc: 0.9445 - val_loss: 0.1798 - val_acc: 0.9339\n",
      "load model ./log/20180707-234645.Multi_LSTM_CNN_v1.word.018.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/30\n",
      "228948/228948 [==============================] - 165s 720us/step - loss: 0.4572 - acc: 0.7840 - val_loss: 0.3461 - val_acc: 0.8575\n",
      "Epoch 2/30\n",
      "228948/228948 [==============================] - 165s 721us/step - loss: 0.3196 - acc: 0.8591 - val_loss: 0.2772 - val_acc: 0.8864\n",
      "Epoch 3/30\n",
      "228948/228948 [==============================] - 162s 709us/step - loss: 0.2729 - acc: 0.8811 - val_loss: 0.2326 - val_acc: 0.9033\n",
      "Epoch 4/30\n",
      "228948/228948 [==============================] - 167s 728us/step - loss: 0.2460 - acc: 0.8940 - val_loss: 0.2250 - val_acc: 0.9057\n",
      "Epoch 5/30\n",
      "228948/228948 [==============================] - 167s 728us/step - loss: 0.2287 - acc: 0.9030 - val_loss: 0.2166 - val_acc: 0.9132\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.2148 - acc: 0.9091 - val_loss: 0.2095 - val_acc: 0.9160\n",
      "Epoch 7/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.2049 - acc: 0.9141 - val_loss: 0.2010 - val_acc: 0.9183\n",
      "Epoch 8/30\n",
      "228948/228948 [==============================] - 159s 695us/step - loss: 0.1960 - acc: 0.9190 - val_loss: 0.2006 - val_acc: 0.9191\n",
      "Epoch 9/30\n",
      "228948/228948 [==============================] - 166s 726us/step - loss: 0.1896 - acc: 0.9207 - val_loss: 0.1904 - val_acc: 0.9238\n",
      "Epoch 10/30\n",
      "228948/228948 [==============================] - 167s 731us/step - loss: 0.1825 - acc: 0.9243 - val_loss: 0.2000 - val_acc: 0.9191\n",
      "Epoch 11/30\n",
      "228948/228948 [==============================] - 169s 738us/step - loss: 0.1774 - acc: 0.9267 - val_loss: 0.1864 - val_acc: 0.9257\n",
      "Epoch 12/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.1741 - acc: 0.9277 - val_loss: 0.2013 - val_acc: 0.9211\n",
      "Epoch 13/30\n",
      "228948/228948 [==============================] - 166s 726us/step - loss: 0.1692 - acc: 0.9296 - val_loss: 0.1799 - val_acc: 0.9265\n",
      "Epoch 14/30\n",
      "228948/228948 [==============================] - 167s 728us/step - loss: 0.1662 - acc: 0.9309 - val_loss: 0.1893 - val_acc: 0.9257\n",
      "Epoch 15/30\n",
      "228948/228948 [==============================] - 167s 731us/step - loss: 0.1622 - acc: 0.9328 - val_loss: 0.1779 - val_acc: 0.9278\n",
      "Epoch 16/30\n",
      "228948/228948 [==============================] - 166s 727us/step - loss: 0.1588 - acc: 0.9344 - val_loss: 0.1956 - val_acc: 0.9249\n",
      "Epoch 17/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.1557 - acc: 0.9357 - val_loss: 0.1822 - val_acc: 0.9289\n",
      "Epoch 18/30\n",
      "228948/228948 [==============================] - 166s 724us/step - loss: 0.1539 - acc: 0.9362 - val_loss: 0.1918 - val_acc: 0.9270\n",
      "Epoch 19/30\n",
      "228948/228948 [==============================] - 167s 729us/step - loss: 0.1506 - acc: 0.9379 - val_loss: 0.1848 - val_acc: 0.9307\n",
      "Epoch 20/30\n",
      "228948/228948 [==============================] - 167s 729us/step - loss: 0.1492 - acc: 0.9388 - val_loss: 0.1817 - val_acc: 0.9304\n",
      "Epoch 21/30\n",
      "228948/228948 [==============================] - 167s 728us/step - loss: 0.1465 - acc: 0.9393 - val_loss: 0.1874 - val_acc: 0.9299\n",
      "Epoch 22/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.1441 - acc: 0.9408 - val_loss: 0.1958 - val_acc: 0.9278\n",
      "Epoch 23/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.1431 - acc: 0.9412 - val_loss: 0.1843 - val_acc: 0.9299\n",
      "load model ./log/20180708-010007.Multi_LSTM_CNN_v1.word.015.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.4584 - acc: 0.7831 - val_loss: 0.3436 - val_acc: 0.8579\n",
      "Epoch 2/30\n",
      "228948/228948 [==============================] - 162s 706us/step - loss: 0.3180 - acc: 0.8589 - val_loss: 0.2778 - val_acc: 0.8851\n",
      "Epoch 3/30\n",
      "228948/228948 [==============================] - 161s 702us/step - loss: 0.2704 - acc: 0.8829 - val_loss: 0.2484 - val_acc: 0.8985\n",
      "Epoch 4/30\n",
      "228948/228948 [==============================] - 163s 711us/step - loss: 0.2432 - acc: 0.8960 - val_loss: 0.2228 - val_acc: 0.9068\n",
      "Epoch 5/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.2252 - acc: 0.9040 - val_loss: 0.2229 - val_acc: 0.9091\n",
      "Epoch 6/30\n",
      "228948/228948 [==============================] - 169s 737us/step - loss: 0.2125 - acc: 0.9103 - val_loss: 0.2173 - val_acc: 0.9117\n",
      "Epoch 7/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.2023 - acc: 0.9153 - val_loss: 0.2049 - val_acc: 0.9147\n",
      "Epoch 8/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1941 - acc: 0.9190 - val_loss: 0.1927 - val_acc: 0.9191\n",
      "Epoch 9/30\n",
      "228948/228948 [==============================] - 166s 723us/step - loss: 0.1869 - acc: 0.9220 - val_loss: 0.1944 - val_acc: 0.9227\n",
      "Epoch 10/30\n",
      "228948/228948 [==============================] - 167s 731us/step - loss: 0.1807 - acc: 0.9253 - val_loss: 0.1923 - val_acc: 0.9232\n",
      "Epoch 11/30\n",
      "228948/228948 [==============================] - 167s 729us/step - loss: 0.1758 - acc: 0.9268 - val_loss: 0.1954 - val_acc: 0.9219\n",
      "Epoch 12/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.1699 - acc: 0.9295 - val_loss: 0.1901 - val_acc: 0.9242\n",
      "Epoch 13/30\n",
      "228948/228948 [==============================] - 167s 729us/step - loss: 0.1672 - acc: 0.9311 - val_loss: 0.1934 - val_acc: 0.9238\n",
      "Epoch 14/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.1628 - acc: 0.9324 - val_loss: 0.1922 - val_acc: 0.9244\n",
      "Epoch 15/30\n",
      "228948/228948 [==============================] - 166s 725us/step - loss: 0.1612 - acc: 0.9328 - val_loss: 0.1878 - val_acc: 0.9277\n",
      "Epoch 16/30\n",
      "228948/228948 [==============================] - 168s 736us/step - loss: 0.1572 - acc: 0.9351 - val_loss: 0.1878 - val_acc: 0.9275\n",
      "Epoch 17/30\n",
      "228948/228948 [==============================] - 168s 732us/step - loss: 0.1538 - acc: 0.9364 - val_loss: 0.1839 - val_acc: 0.9283\n",
      "Epoch 18/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1518 - acc: 0.9373 - val_loss: 0.1834 - val_acc: 0.9281\n",
      "Epoch 19/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.1505 - acc: 0.9381 - val_loss: 0.1970 - val_acc: 0.9281\n",
      "Epoch 20/30\n",
      "228948/228948 [==============================] - 168s 732us/step - loss: 0.1476 - acc: 0.9392 - val_loss: 0.1846 - val_acc: 0.9303\n",
      "Epoch 21/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1447 - acc: 0.9400 - val_loss: 0.1955 - val_acc: 0.9274\n",
      "Epoch 22/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1434 - acc: 0.9406 - val_loss: 0.1881 - val_acc: 0.9299\n",
      "Epoch 23/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.1404 - acc: 0.9424 - val_loss: 0.1946 - val_acc: 0.9287\n",
      "Epoch 24/30\n",
      "228948/228948 [==============================] - 168s 732us/step - loss: 0.1391 - acc: 0.9431 - val_loss: 0.1843 - val_acc: 0.9283\n",
      "Epoch 25/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.1372 - acc: 0.9439 - val_loss: 0.1863 - val_acc: 0.9312\n",
      "Epoch 26/30\n",
      "228948/228948 [==============================] - 169s 738us/step - loss: 0.1355 - acc: 0.9443 - val_loss: 0.1873 - val_acc: 0.9309\n",
      "load model ./log/20180708-020528.Multi_LSTM_CNN_v1.word.018.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/30\n",
      "228948/228948 [==============================] - 165s 720us/step - loss: 0.4546 - acc: 0.7853 - val_loss: 0.3503 - val_acc: 0.8526\n",
      "Epoch 2/30\n",
      "228948/228948 [==============================] - 158s 692us/step - loss: 0.3165 - acc: 0.8598 - val_loss: 0.2682 - val_acc: 0.8867\n",
      "Epoch 3/30\n",
      "228948/228948 [==============================] - 167s 728us/step - loss: 0.2703 - acc: 0.8828 - val_loss: 0.2574 - val_acc: 0.8967\n",
      "Epoch 4/30\n",
      "228948/228948 [==============================] - 168s 732us/step - loss: 0.2432 - acc: 0.8957 - val_loss: 0.2154 - val_acc: 0.9106\n",
      "Epoch 5/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.2246 - acc: 0.9043 - val_loss: 0.2380 - val_acc: 0.9014\n",
      "Epoch 6/30\n",
      "228948/228948 [==============================] - 168s 732us/step - loss: 0.2132 - acc: 0.9106 - val_loss: 0.2039 - val_acc: 0.9147\n",
      "Epoch 7/30\n",
      "228948/228948 [==============================] - 169s 738us/step - loss: 0.2018 - acc: 0.9149 - val_loss: 0.1904 - val_acc: 0.9227\n",
      "Epoch 8/30\n",
      "228948/228948 [==============================] - 167s 731us/step - loss: 0.1950 - acc: 0.9185 - val_loss: 0.1940 - val_acc: 0.9208\n",
      "Epoch 9/30\n",
      "228948/228948 [==============================] - 167s 729us/step - loss: 0.1877 - acc: 0.9215 - val_loss: 0.1888 - val_acc: 0.9253\n",
      "Epoch 10/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1819 - acc: 0.9240 - val_loss: 0.1854 - val_acc: 0.9248\n",
      "Epoch 11/30\n",
      "228948/228948 [==============================] - 167s 731us/step - loss: 0.1766 - acc: 0.9266 - val_loss: 0.1828 - val_acc: 0.9285\n",
      "Epoch 12/30\n",
      "228948/228948 [==============================] - 161s 704us/step - loss: 0.1724 - acc: 0.9283 - val_loss: 0.1885 - val_acc: 0.9261\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1684 - acc: 0.9298 - val_loss: 0.1880 - val_acc: 0.9278\n",
      "Epoch 14/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.1636 - acc: 0.9317 - val_loss: 0.1852 - val_acc: 0.9296\n",
      "Epoch 15/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.1615 - acc: 0.9335 - val_loss: 0.1829 - val_acc: 0.9298\n",
      "Epoch 16/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.1571 - acc: 0.9351 - val_loss: 0.1899 - val_acc: 0.9275\n",
      "Epoch 17/30\n",
      "228948/228948 [==============================] - 169s 740us/step - loss: 0.1553 - acc: 0.9359 - val_loss: 0.1886 - val_acc: 0.9302\n",
      "Epoch 18/30\n",
      "228948/228948 [==============================] - 167s 728us/step - loss: 0.1519 - acc: 0.9368 - val_loss: 0.1863 - val_acc: 0.9290\n",
      "Epoch 19/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1487 - acc: 0.9383 - val_loss: 0.1809 - val_acc: 0.9329\n",
      "Epoch 20/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.1469 - acc: 0.9393 - val_loss: 0.1872 - val_acc: 0.9294\n",
      "Epoch 21/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1451 - acc: 0.9405 - val_loss: 0.1810 - val_acc: 0.9336\n",
      "Epoch 22/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.1419 - acc: 0.9417 - val_loss: 0.2005 - val_acc: 0.9248\n",
      "Epoch 23/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.1411 - acc: 0.9421 - val_loss: 0.1856 - val_acc: 0.9322\n",
      "Epoch 24/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1394 - acc: 0.9422 - val_loss: 0.1847 - val_acc: 0.9320\n",
      "Epoch 25/30\n",
      "228948/228948 [==============================] - 168s 736us/step - loss: 0.1367 - acc: 0.9437 - val_loss: 0.1840 - val_acc: 0.9314\n",
      "Epoch 26/30\n",
      "228948/228948 [==============================] - 167s 728us/step - loss: 0.1364 - acc: 0.9438 - val_loss: 0.1812 - val_acc: 0.9339\n",
      "Epoch 27/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.1334 - acc: 0.9451 - val_loss: 0.1887 - val_acc: 0.9323\n",
      "load model ./log/20180708-031925.Multi_LSTM_CNN_v1.word.019.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/30\n",
      "228948/228948 [==============================] - 170s 742us/step - loss: 0.4549 - acc: 0.7851 - val_loss: 0.3212 - val_acc: 0.8628\n",
      "Epoch 2/30\n",
      "228948/228948 [==============================] - 159s 694us/step - loss: 0.3155 - acc: 0.8608 - val_loss: 0.2780 - val_acc: 0.8875\n",
      "Epoch 3/30\n",
      "228948/228948 [==============================] - 166s 725us/step - loss: 0.2708 - acc: 0.8824 - val_loss: 0.2351 - val_acc: 0.9059\n",
      "Epoch 4/30\n",
      "228948/228948 [==============================] - 167s 729us/step - loss: 0.2441 - acc: 0.8959 - val_loss: 0.2156 - val_acc: 0.9103\n",
      "Epoch 5/30\n",
      "228948/228948 [==============================] - 162s 709us/step - loss: 0.2259 - acc: 0.9043 - val_loss: 0.2109 - val_acc: 0.9158\n",
      "Epoch 6/30\n",
      "228948/228948 [==============================] - 164s 716us/step - loss: 0.2122 - acc: 0.9101 - val_loss: 0.1994 - val_acc: 0.9204\n",
      "Epoch 7/30\n",
      "228948/228948 [==============================] - 159s 694us/step - loss: 0.2012 - acc: 0.9158 - val_loss: 0.1955 - val_acc: 0.9228\n",
      "Epoch 8/30\n",
      "228948/228948 [==============================] - 165s 720us/step - loss: 0.1933 - acc: 0.9190 - val_loss: 0.2056 - val_acc: 0.9179\n",
      "Epoch 9/30\n",
      "228948/228948 [==============================] - 169s 737us/step - loss: 0.1870 - acc: 0.9226 - val_loss: 0.2007 - val_acc: 0.9235\n",
      "Epoch 10/30\n",
      "228948/228948 [==============================] - 167s 732us/step - loss: 0.1803 - acc: 0.9245 - val_loss: 0.1878 - val_acc: 0.9289\n",
      "Epoch 11/30\n",
      "228948/228948 [==============================] - 164s 716us/step - loss: 0.1764 - acc: 0.9267 - val_loss: 0.1980 - val_acc: 0.9248\n",
      "Epoch 12/30\n",
      "228948/228948 [==============================] - 167s 729us/step - loss: 0.1709 - acc: 0.9292 - val_loss: 0.1884 - val_acc: 0.9290\n",
      "Epoch 13/30\n",
      "228948/228948 [==============================] - 166s 726us/step - loss: 0.1675 - acc: 0.9304 - val_loss: 0.1893 - val_acc: 0.9286\n",
      "Epoch 14/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.1634 - acc: 0.9323 - val_loss: 0.1829 - val_acc: 0.9300\n",
      "Epoch 15/30\n",
      "228948/228948 [==============================] - 169s 737us/step - loss: 0.1584 - acc: 0.9347 - val_loss: 0.1856 - val_acc: 0.9301\n",
      "Epoch 16/30\n",
      "228948/228948 [==============================] - 167s 731us/step - loss: 0.1571 - acc: 0.9355 - val_loss: 0.1827 - val_acc: 0.9323\n",
      "Epoch 17/30\n",
      "228948/228948 [==============================] - 168s 732us/step - loss: 0.1536 - acc: 0.9362 - val_loss: 0.1854 - val_acc: 0.9319\n",
      "Epoch 18/30\n",
      "228948/228948 [==============================] - 168s 736us/step - loss: 0.1516 - acc: 0.9375 - val_loss: 0.1929 - val_acc: 0.9298\n",
      "Epoch 19/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1483 - acc: 0.9385 - val_loss: 0.1804 - val_acc: 0.9336\n",
      "Epoch 20/30\n",
      "228948/228948 [==============================] - 165s 723us/step - loss: 0.1471 - acc: 0.9389 - val_loss: 0.1868 - val_acc: 0.9313\n",
      "Epoch 21/30\n",
      "228948/228948 [==============================] - 167s 729us/step - loss: 0.1446 - acc: 0.9405 - val_loss: 0.1817 - val_acc: 0.9342\n",
      "Epoch 22/30\n",
      "228948/228948 [==============================] - 169s 738us/step - loss: 0.1418 - acc: 0.9411 - val_loss: 0.1813 - val_acc: 0.9327\n",
      "Epoch 23/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.1396 - acc: 0.9424 - val_loss: 0.1738 - val_acc: 0.9338\n",
      "Epoch 24/30\n",
      "228948/228948 [==============================] - 167s 731us/step - loss: 0.1394 - acc: 0.9427 - val_loss: 0.1860 - val_acc: 0.9318\n",
      "Epoch 25/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.1360 - acc: 0.9442 - val_loss: 0.1813 - val_acc: 0.9320\n",
      "Epoch 26/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.1346 - acc: 0.9447 - val_loss: 0.1830 - val_acc: 0.9332\n",
      "Epoch 27/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1332 - acc: 0.9451 - val_loss: 0.1804 - val_acc: 0.9344\n",
      "Epoch 28/30\n",
      "228948/228948 [==============================] - 169s 737us/step - loss: 0.1309 - acc: 0.9464 - val_loss: 0.1900 - val_acc: 0.9327\n",
      "Epoch 29/30\n",
      "228948/228948 [==============================] - 169s 737us/step - loss: 0.1309 - acc: 0.9462 - val_loss: 0.1849 - val_acc: 0.9323\n",
      "Epoch 30/30\n",
      "228948/228948 [==============================] - 168s 732us/step - loss: 0.1277 - acc: 0.9471 - val_loss: 0.1995 - val_acc: 0.9273\n",
      "load model ./log/20180708-043614.Multi_LSTM_CNN_v1.word.023.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/30\n",
      "228948/228948 [==============================] - 166s 724us/step - loss: 0.4510 - acc: 0.7882 - val_loss: 0.3387 - val_acc: 0.8598\n",
      "Epoch 2/30\n",
      "228948/228948 [==============================] - 164s 716us/step - loss: 0.3154 - acc: 0.8604 - val_loss: 0.2612 - val_acc: 0.8896\n",
      "Epoch 3/30\n",
      "228948/228948 [==============================] - 167s 728us/step - loss: 0.2686 - acc: 0.8840 - val_loss: 0.2331 - val_acc: 0.9020\n",
      "Epoch 4/30\n",
      "228948/228948 [==============================] - 167s 728us/step - loss: 0.2424 - acc: 0.8961 - val_loss: 0.2384 - val_acc: 0.9018\n",
      "Epoch 5/30\n",
      "228948/228948 [==============================] - 170s 742us/step - loss: 0.2245 - acc: 0.9046 - val_loss: 0.2075 - val_acc: 0.9138\n",
      "Epoch 6/30\n",
      "228948/228948 [==============================] - 170s 741us/step - loss: 0.2124 - acc: 0.9103 - val_loss: 0.2043 - val_acc: 0.9172\n",
      "Epoch 7/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.2012 - acc: 0.9153 - val_loss: 0.2086 - val_acc: 0.9156\n",
      "Epoch 8/30\n",
      "228948/228948 [==============================] - 168s 732us/step - loss: 0.1926 - acc: 0.9192 - val_loss: 0.2038 - val_acc: 0.9174\n",
      "Epoch 9/30\n",
      "228948/228948 [==============================] - 164s 716us/step - loss: 0.1865 - acc: 0.9225 - val_loss: 0.2008 - val_acc: 0.9218\n",
      "Epoch 10/30\n",
      "228948/228948 [==============================] - 164s 716us/step - loss: 0.1802 - acc: 0.9248 - val_loss: 0.1869 - val_acc: 0.9245\n",
      "Epoch 11/30\n",
      "228948/228948 [==============================] - 159s 695us/step - loss: 0.1750 - acc: 0.9275 - val_loss: 0.1914 - val_acc: 0.9247\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12/30\n",
      "228948/228948 [==============================] - 159s 695us/step - loss: 0.1703 - acc: 0.9293 - val_loss: 0.1852 - val_acc: 0.9277\n",
      "Epoch 13/30\n",
      "228948/228948 [==============================] - 159s 692us/step - loss: 0.1662 - acc: 0.9314 - val_loss: 0.1881 - val_acc: 0.9251\n",
      "Epoch 14/30\n",
      "228948/228948 [==============================] - 161s 701us/step - loss: 0.1637 - acc: 0.9321 - val_loss: 0.1931 - val_acc: 0.9233\n",
      "Epoch 15/30\n",
      "228948/228948 [==============================] - 169s 739us/step - loss: 0.1591 - acc: 0.9342 - val_loss: 0.1828 - val_acc: 0.9273\n",
      "Epoch 16/30\n",
      "228948/228948 [==============================] - 168s 732us/step - loss: 0.1563 - acc: 0.9352 - val_loss: 0.1891 - val_acc: 0.9259\n",
      "Epoch 17/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.1542 - acc: 0.9365 - val_loss: 0.1928 - val_acc: 0.9283\n",
      "Epoch 18/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.1499 - acc: 0.9379 - val_loss: 0.1943 - val_acc: 0.9268\n",
      "Epoch 19/30\n",
      "228948/228948 [==============================] - 169s 739us/step - loss: 0.1483 - acc: 0.9388 - val_loss: 0.1890 - val_acc: 0.9276\n",
      "Epoch 20/30\n",
      "228948/228948 [==============================] - 167s 729us/step - loss: 0.1465 - acc: 0.9403 - val_loss: 0.1911 - val_acc: 0.9253\n",
      "Epoch 21/30\n",
      "228948/228948 [==============================] - 169s 736us/step - loss: 0.1435 - acc: 0.9412 - val_loss: 0.1841 - val_acc: 0.9298\n",
      "Epoch 22/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.1421 - acc: 0.9415 - val_loss: 0.2000 - val_acc: 0.9263\n",
      "Epoch 23/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.1398 - acc: 0.9427 - val_loss: 0.1872 - val_acc: 0.9302\n",
      "load model ./log/20180708-060114.Multi_LSTM_CNN_v1.word.015.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/30\n",
      "228948/228948 [==============================] - 173s 756us/step - loss: 0.4508 - acc: 0.7881 - val_loss: 0.3316 - val_acc: 0.8588\n",
      "Epoch 2/30\n",
      "228948/228948 [==============================] - 165s 720us/step - loss: 0.3139 - acc: 0.8598 - val_loss: 0.2602 - val_acc: 0.8893\n",
      "Epoch 3/30\n",
      "228948/228948 [==============================] - 170s 742us/step - loss: 0.2695 - acc: 0.8830 - val_loss: 0.2319 - val_acc: 0.9044\n",
      "Epoch 4/30\n",
      "228948/228948 [==============================] - 169s 738us/step - loss: 0.2430 - acc: 0.8953 - val_loss: 0.2185 - val_acc: 0.9085\n",
      "Epoch 5/30\n",
      "228948/228948 [==============================] - 169s 737us/step - loss: 0.2261 - acc: 0.9045 - val_loss: 0.2077 - val_acc: 0.9114\n",
      "Epoch 6/30\n",
      "228948/228948 [==============================] - 169s 740us/step - loss: 0.2114 - acc: 0.9101 - val_loss: 0.1978 - val_acc: 0.9197\n",
      "Epoch 7/30\n",
      "228948/228948 [==============================] - 169s 736us/step - loss: 0.2033 - acc: 0.9151 - val_loss: 0.1939 - val_acc: 0.9202\n",
      "Epoch 8/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.1938 - acc: 0.9187 - val_loss: 0.1897 - val_acc: 0.9234\n",
      "Epoch 9/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1864 - acc: 0.9220 - val_loss: 0.1827 - val_acc: 0.9280\n",
      "Epoch 10/30\n",
      "228948/228948 [==============================] - 170s 741us/step - loss: 0.1821 - acc: 0.9248 - val_loss: 0.1886 - val_acc: 0.9256\n",
      "Epoch 11/30\n",
      "228948/228948 [==============================] - 168s 735us/step - loss: 0.1765 - acc: 0.9266 - val_loss: 0.1786 - val_acc: 0.9264\n",
      "Epoch 12/30\n",
      "228948/228948 [==============================] - 167s 730us/step - loss: 0.1717 - acc: 0.9285 - val_loss: 0.1850 - val_acc: 0.9290\n",
      "Epoch 13/30\n",
      "228948/228948 [==============================] - 170s 740us/step - loss: 0.1673 - acc: 0.9307 - val_loss: 0.1869 - val_acc: 0.9281\n",
      "Epoch 14/30\n",
      "228948/228948 [==============================] - 168s 734us/step - loss: 0.1634 - acc: 0.9322 - val_loss: 0.1786 - val_acc: 0.9302\n",
      "Epoch 15/30\n",
      "228948/228948 [==============================] - 167s 727us/step - loss: 0.1601 - acc: 0.9336 - val_loss: 0.1806 - val_acc: 0.9290\n",
      "Epoch 16/30\n",
      "228948/228948 [==============================] - 170s 743us/step - loss: 0.1575 - acc: 0.9348 - val_loss: 0.1771 - val_acc: 0.9315\n",
      "Epoch 17/30\n",
      "228948/228948 [==============================] - 169s 738us/step - loss: 0.1542 - acc: 0.9366 - val_loss: 0.1721 - val_acc: 0.9318\n",
      "Epoch 18/30\n",
      "228948/228948 [==============================] - 169s 739us/step - loss: 0.1501 - acc: 0.9376 - val_loss: 0.1784 - val_acc: 0.9314\n",
      "Epoch 19/30\n",
      "228948/228948 [==============================] - 169s 737us/step - loss: 0.1493 - acc: 0.9385 - val_loss: 0.1831 - val_acc: 0.9287\n",
      "Epoch 20/30\n",
      "228948/228948 [==============================] - 169s 738us/step - loss: 0.1469 - acc: 0.9396 - val_loss: 0.1846 - val_acc: 0.9305\n",
      "Epoch 21/30\n",
      "228948/228948 [==============================] - 170s 743us/step - loss: 0.1449 - acc: 0.9405 - val_loss: 0.1811 - val_acc: 0.9318\n",
      "Epoch 22/30\n",
      "228948/228948 [==============================] - 170s 745us/step - loss: 0.1430 - acc: 0.9415 - val_loss: 0.1829 - val_acc: 0.9303\n",
      "Epoch 23/30\n",
      "228948/228948 [==============================] - 169s 738us/step - loss: 0.1403 - acc: 0.9421 - val_loss: 0.1792 - val_acc: 0.9328\n",
      "Epoch 24/30\n",
      "228948/228948 [==============================] - 168s 733us/step - loss: 0.1379 - acc: 0.9433 - val_loss: 0.1776 - val_acc: 0.9338\n",
      "Epoch 25/30\n",
      "228948/228948 [==============================] - 170s 741us/step - loss: 0.1368 - acc: 0.9439 - val_loss: 0.1824 - val_acc: 0.9317\n",
      "load model ./log/20180708-070633.Multi_LSTM_CNN_v1.word.017.hdf5\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "best_results = []\n",
    "last_results = []\n",
    "\n",
    "for i, (train_index, dev_index) in enumerate(StratifiedKFold(n_splits=10).split(X=word1, y=label)):  # word/char switch\n",
    "    train_x1, train_x2, train_y = word1[train_index, :], word2[train_index, :], label[train_index]  # word/char switch\n",
    "    dev_x1, dev_x2, dev_y = word1[dev_index, :], word2[dev_index, :], label[dev_index]  # word/char switch\n",
    "    \n",
    "    input1 = Input(shape=(SEQ_LEN,), dtype=\"int32\")\n",
    "    input2 = Input(shape=(SEQ_LEN,), dtype=\"int32\")\n",
    "\n",
    "    embedding_layer = Embedding(\n",
    "        input_dim=word_embedding_data.shape[0],  # word/char switch\n",
    "        output_dim=word_embedding_data.shape[1],  # word/char switch\n",
    "        weights=[word_embedding_data],  # word/char switch\n",
    "        input_length=SEQ_LEN,\n",
    "        trainable=False\n",
    "    )\n",
    "    \n",
    "    vector1 = embedding_layer(input1)\n",
    "    vector2 = embedding_layer(input2)\n",
    "    \n",
    "    lstm_layer1 = LSTM(LSTM_SIZE1, dropout=LSTM_DROP_RATE, recurrent_dropout=LSTM_DROP_RATE, return_sequences=True)\n",
    "    layer1a = lstm_layer1(vector1)\n",
    "    layer1a = Dropout(LSTM_DROP_RATE)(layer1a)\n",
    "    layer1b = lstm_layer1(vector2)\n",
    "    layer1b = Dropout(LSTM_DROP_RATE)(layer1b)\n",
    "\n",
    "    lstm_layer2 = LSTM(LSTM_SIZE2, dropout=LSTM_DROP_RATE, recurrent_dropout=LSTM_DROP_RATE, return_sequences=True)\n",
    "    layer2a = lstm_layer2(layer1a)\n",
    "    layer2b = lstm_layer2(layer1b)\n",
    "#     # 每个序列片拼接对应的原始embedding向量\n",
    "#     layer2a = concatenate([vector1, layer2a])\n",
    "#     layer2b = concatenate([vector2, layer2b])\n",
    "    \n",
    "    conv1a, conv1b = cnn_layer3(layer2a, layer2b, filters=CONV_LEN1, kernel_size=1)\n",
    "    conv2a, conv2b = cnn_layer3(layer2a, layer2b, filters=CONV_LEN2, kernel_size=2)\n",
    "    conv3a, conv3b = cnn_layer3(layer2a, layer2b, filters=CONV_LEN3, kernel_size=3)\n",
    "    conv4a, conv4b = cnn_layer3(layer2a, layer2b, filters=CONV_LEN4, kernel_size=4)\n",
    "    conv5a, conv5b = cnn_layer3(layer2a, layer2b, filters=CONV_LEN5, kernel_size=5)\n",
    "    conv6a, conv6b = cnn_layer3(layer2a, layer2b, filters=CONV_LEN6, kernel_size=6)\n",
    "    \n",
    "    merge_a = concatenate([conv1a, conv2a, conv3a, conv4a, conv5a, conv6a])\n",
    "    merge_b = concatenate([conv1b, conv2b, conv3b, conv4b, conv5b, conv6b])\n",
    "    diff = Lambda(lambda x: K.abs(x[0] - x[1]))([merge_a, merge_b])\n",
    "    mult = Lambda(lambda x: x[0] * x[1])([merge_a, merge_b])\n",
    "    merge = concatenate([diff, mult])\n",
    "    \n",
    "    x = Dropout(DROP_RATE)(merge)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(DENSE_SIZE1, activation=\"relu\")(x)\n",
    "    x = Dropout(DROP_RATE)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(DENSE_SIZE2, activation=\"relu\")(x)\n",
    "    x = Dropout(DROP_RATE)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    pred = Dense(1, activation=\"sigmoid\")(x)\n",
    "    \n",
    "    model = Model(inputs=[input1, input2], outputs=pred)\n",
    "    model.compile(\n",
    "        optimizer=\"nadam\",\n",
    "        loss=\"binary_crossentropy\",\n",
    "        metrics=[\"acc\"]\n",
    "    )\n",
    "    \n",
    "    early_stopping = EarlyStopping(\"val_loss\", patience=PATIENCE)\n",
    "    check_point = ModelCheckpoint(\n",
    "        \"./log/%s.Multi_LSTM_CNN_v1.word.{epoch:03d}.hdf5\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "        monitor=\"val_loss\",\n",
    "        save_best_only=True,\n",
    "    )\n",
    "    \n",
    "    fit_res = model.fit(\n",
    "        x=[train_x1, train_x2],\n",
    "        y=train_y,\n",
    "        batch_size=BATCH_SIZE,\n",
    "        epochs=NUM_EPOCHES,\n",
    "        validation_data=([dev_x1, dev_x2], dev_y),\n",
    "        shuffle=True,\n",
    "        callbacks=[early_stopping, check_point]\n",
    "    )\n",
    "    \n",
    "    pred_last = model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)  # word/char switch\n",
    "    last_results.append(pd.DataFrame(pred_last, columns=[\"y_pre\"]))\n",
    "    \n",
    "    print(\"load model %s\" % (glob(\"./log/*.hdf5\")[-1].replace(\"\\\\\", \"/\"),))\n",
    "    model.load_weights(glob(\"./log/*.hdf5\")[-1].replace(\"\\\\\", \"/\"))\n",
    "    pred_best = model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)  # word/char switch\n",
    "    best_results.append(pd.DataFrame(pred_best, columns=[\"y_pre\"]))\n",
    "\n",
    "pd.DataFrame(pd.concat(last_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "    \"./result/%s-Multi_LSTM_CNN_v1_word_last.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "    index=False\n",
    ")\n",
    "pd.DataFrame(pd.concat(best_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "    \"./result/%s-Multi_LSTM_CNN_v1_word_best.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "    index=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
